{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [200]>"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Python爬虫\n",
    "\n",
    "# 导入request处理请求\n",
    "import requests\n",
    "\n",
    "# 待爬取JSON的URL\n",
    "url=\"https://www.thepaper.cn/channel_25950\"\n",
    "\n",
    "# GET请求URL（因为我们访问的时候也是用的GET method）\n",
    "resp=requests.get(url=url)\n",
    "resp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "lxml.etree._Element"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(get_html(url))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Element html at 0x7f3dc8ce8f80>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from lxml import etree\n",
    "\n",
    "html = etree.HTML(resp.content)\n",
    "html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: {'title': '图集｜包机“出海抢订单”！多地组团打响外贸保卫战',\n",
       "  'img': 'https://imagecloud.thepaper.cn/thepaper/image/228/961/897.jpg?x-oss-process=image/resize,w_332',\n",
       "  'come': '快看',\n",
       "  'time': '1小时前',\n",
       "  'link': 'https://www.thepaper.cn/newsDetail_forward_21096249'},\n",
       " 2: {'title': '律界“邱喻大战”落幕：当事一方去世5年后，重审法院终止审理',\n",
       "  'img': 'https://imagecloud.thepaper.cn/thepaper/image/229/7/3.jpg?x-oss-process=image/resize,w_332',\n",
       "  'come': '一号专案',\n",
       "  'time': '2小时前',\n",
       "  'link': 'https://www.thepaper.cn/newsDetail_forward_21100246'},\n",
       " 3: {'title': '央视热评丨以司法保障捍卫英烈权益，筑牢信仰之基',\n",
       "  'img': 'https://imagecloud.thepaper.cn/thepaper/image/229/103/768.jpg?x-oss-process=image/resize,w_332',\n",
       "  'come': '舆论场',\n",
       "  'time': '17分钟前',\n",
       "  'link': 'https://www.thepaper.cn/newsDetail_forward_21109788'},\n",
       " 4: {'title': '国家铁路1至11月份货物运输持续保持高位运行',\n",
       "  'img': 'https://imagecloud.thepaper.cn/thepaper/image/229/104/19.png?x-oss-process=image/resize,w_332',\n",
       "  'come': '中国政库',\n",
       "  'time': '22分钟前',\n",
       "  'link': 'https://www.thepaper.cn/newsDetail_forward_21109818'},\n",
       " 5: {'title': '魏凤和在上合组织、独联体成员国国防部长会议上发表视频讲话',\n",
       "  'img': 'https://imagecloud.thepaper.cn/thepaper/image/229/102/347.jpg?x-oss-process=image/resize,w_332',\n",
       "  'come': '澎湃防务',\n",
       "  'time': '26分钟前',\n",
       "  'link': 'https://www.thepaper.cn/newsDetail_forward_21109635'},\n",
       " 6: {'title': '国家卫健委：昨日新增本土3034+10551',\n",
       "  'img': 'https://imagecloud.thepaper.cn/thepaper/image/229/103/734.jpg?x-oss-process=image/resize,w_332',\n",
       "  'come': '绿政公署',\n",
       "  'time': '27分钟前',\n",
       "  'link': 'https://www.thepaper.cn/newsDetail_forward_21109784'},\n",
       " 7: {'title': '早安·世界｜英国皇家邮政工人再次举行罢工',\n",
       "  'img': 'https://imagecloud.thepaper.cn/thepaper/image/229/101/29.jpg?x-oss-process=image/resize,w_332',\n",
       "  'come': '快看',\n",
       "  'time': '32分钟前',\n",
       "  'link': 'https://www.thepaper.cn/newsDetail_forward_21109591'},\n",
       " 8: {'title': '上海昨日新增本土确诊13例、无症状感染者259例',\n",
       "  'img': 'https://imagecloud.thepaper.cn/thepaper/image/229/100/816.jpg?x-oss-process=image/resize,w_332',\n",
       "  'come': '浦江头条',\n",
       "  'time': '47分钟前',\n",
       "  'link': 'https://www.thepaper.cn/newsDetail_forward_21109572'},\n",
       " 9: {'title': '进球英雄｜梅西传射建功！阿根廷2-2荷兰，以点球惊险晋级',\n",
       "  'img': 'https://imagecloud.thepaper.cn/thepaper/image/229/100/542.jpg?x-oss-process=image/resize,w_332',\n",
       "  'come': '快看',\n",
       "  'time': '1小时前',\n",
       "  'link': 'https://www.thepaper.cn/newsDetail_forward_21109528'},\n",
       " 10: {'title': '射手卡｜昨夜，男儿有泪！内马尔伤心欲绝，五星巴西抱憾回家',\n",
       "  'img': 'https://imagecloud.thepaper.cn/thepaper/image/229/56/898.jpg?x-oss-process=image/resize,w_332',\n",
       "  'come': '快看',\n",
       "  'time': '1小时前',\n",
       "  'link': 'https://www.thepaper.cn/newsDetail_forward_21102190'}}"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dct = dict()\n",
    "prefix = 'https://www.thepaper.cn'\n",
    "for i in range(1, 11):\n",
    "    title = html.xpath(\n",
    "        f'/html/body/div[1]/main/div[3]/div[1]/div[2]/div[2]/div/div[1]/div[{i}]/div/div/div/div[1]/a/h2/text()')\n",
    "    img = html.xpath(\n",
    "        f'/html/body/div[1]/main/div[3]/div[1]/div[2]/div[2]/div/div[1]/div[{i}]/div/div/div/div[1]/a/div/img/@src')\n",
    "    come = html.xpath(\n",
    "        f'/html/body/div[1]/main/div[3]/div[1]/div[2]/div[2]/div/div[1]/div[{i}]/div/div/div/div[2]/div/p[1]/span[1]/a/text()')\n",
    "    time = html.xpath(\n",
    "        f'/html/body/div[1]/main/div[3]/div[1]/div[2]/div[2]/div/div[1]/div[{i}]/div/div/div/div[2]/div/p[1]/span[2]/text()')\n",
    "    link = html.xpath(\n",
    "        f'/html/body/div[1]/main/div[3]/div[1]/div[2]/div[2]/div/div[1]/div[{i}]/div/div/div/div[1]/a/@href')\n",
    "\n",
    "    dct[i] = {\"title\": title[0], \"img\": img[0],\n",
    "              \"come\": come[0], \"time\": time[0],\"link\":prefix+link[0]}\n",
    "\n",
    "dct\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://imagecloud.thepaper.cn/thepaper/image/228/961/897.jpg?x-oss-process=image/resize,w_332 .jpg\n",
      "https://imagecloud.thepaper.cn/thepaper/image/229/7/3.jpg?x-oss-process=image/resize,w_332 .jpg\n",
      "https://imagecloud.thepaper.cn/thepaper/image/229/103/768.jpg?x-oss-process=image/resize,w_332 .jpg\n",
      "https://imagecloud.thepaper.cn/thepaper/image/229/104/19.png?x-oss-process=image/resize,w_332 .png\n",
      "https://imagecloud.thepaper.cn/thepaper/image/229/102/347.jpg?x-oss-process=image/resize,w_332 .jpg\n",
      "https://imagecloud.thepaper.cn/thepaper/image/229/103/734.jpg?x-oss-process=image/resize,w_332 .jpg\n",
      "https://imagecloud.thepaper.cn/thepaper/image/229/101/29.jpg?x-oss-process=image/resize,w_332 .jpg\n",
      "https://imagecloud.thepaper.cn/thepaper/image/229/100/816.jpg?x-oss-process=image/resize,w_332 .jpg\n",
      "https://imagecloud.thepaper.cn/thepaper/image/229/100/542.jpg?x-oss-process=image/resize,w_332 .jpg\n",
      "https://imagecloud.thepaper.cn/thepaper/image/229/56/898.jpg?x-oss-process=image/resize,w_332 .jpg\n"
     ]
    }
   ],
   "source": [
    "# 下载图片\n",
    "for j,i in dct.items():\n",
    "  url=(str)(i['img'])\n",
    "  fix=url[url.rfind('.'):url.rfind('?')]\n",
    "  print(url,fix)\n",
    "  img=requests.get(url)\n",
    "  with open(f'../data/img/{j}{fix}','wb') as f:\n",
    "    f.write(img.content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dict2xml(dct: dict):\n",
    "    \"\"\"\n",
    "    python字典转xml\n",
    "    参数 dct: python字典\n",
    "    \"\"\"\n",
    "    with open('../data/info.xml', 'w') as f:\n",
    "        f.write('<?xml version=\"1.0\" encoding=\"UTF-8\"?>')\n",
    "        f.write('<infomation>')\n",
    "        for j, i in dct.items():\n",
    "            url=i['img']\n",
    "            fix=url[url.rfind('.'):url.rfind('?')]\n",
    "            # print(i)\n",
    "            f.write(f\"\"\"\n",
    "            <info>\n",
    "            <title>{i['title']}</title>\n",
    "            <img>{j}{fix}</img>\n",
    "            <come>{i['come']}</come>\n",
    "            <time>{i['time']}</time>\n",
    "            <link>{i['link']}</link>\n",
    "            </info>\n",
    "             \"\"\")\n",
    "        f.write('</infomation>')\n",
    "\n",
    "\n",
    "dict2xml(dct)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dict2json(dct: dict):\n",
    "    \"\"\"\n",
    "    python字典转json\n",
    "    参数 dct: python字典\n",
    "    \"\"\"\n",
    "    with open('../data/info.json', 'w') as f:\n",
    "        f.write('{\"infomation\":[')\n",
    "        for j, i in dct.items():\n",
    "            url=i['img']\n",
    "            fix=url[url.rfind('.'):url.rfind('?')]\n",
    "            f.write(f\"\"\"\n",
    "            {{\n",
    "            \"title\":\"{i['title']}\",\n",
    "            \"img\":\"{j}{fix}\",\n",
    "            \"come\":\"{i['come']}\",\n",
    "            \"time\":\"{i['time']}\",\n",
    "            \"link\":\"{i['link']}\"\n",
    "            }}{',' if j!=len(dct)else ''}\n",
    "              \"\"\")\n",
    "        f.write(']}')\n",
    "\n",
    "\n",
    "dict2json(dct)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.12 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "4c2b8563932708ac4f64e3bc4797ea8eb43fc7530baa7f01daadf5b46d7b1fd4"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
